{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Single channel sleep scoring comparison sheet\n",
    "\n",
    "Calculations carried out to fairly compare papers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initial formatting:\n",
    "* Confusion matrices are gathered from original articles and formatted in the order W, S1, S2, S3(&4), R, transposed if necessary\n",
    "* If only the 6-class confmat is available, classes S3 and S4 are merged\n",
    "* Some studies leave many more W stages than other stages which articifially increases claimed performance. For those studies, we bring down the number of Wake epochs (according to expert label) to the cardinal of the second biggest class.\n",
    "\n",
    "Then we calculate some metrics for each of the studies:\n",
    "* accuracy\n",
    "* kappa\n",
    "* f1-micro\n",
    "* f1-macro\n",
    "\n",
    "And aggregate everything in a pandas dataframe which we can export to the latex article. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import sklearn.metrics\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# order: W, S1, S2, S3(&4), REM\n",
    "# if matrix in article is not given in this order, we copy it as such and reorder it later to avoid errors\n",
    "def reorder(cm, new_order=[4, 0, 1, 2, 3]):\n",
    "    cm = cm[new_order, :]\n",
    "    cm = cm[:, new_order]\n",
    "    return cm\n",
    "\n",
    "# of only 6-class CM is available, transform it into 5-class by merging 3 and 4\n",
    "def merge(cm, indices_keep=[0, 1, 4, 5], indices_merge=[2, 3]):\n",
    "    # lines\n",
    "    cm_keep = cm[indices_keep, :]\n",
    "    cm_ind = cm[indices_merge, :]\n",
    "    cm_ind = np.sum(cm_ind, axis=0, keepdims=True)\n",
    "    ind_toinsert = np.min(np.where((np.array(indices_keep)>indices_merge[0])==1)[0])\n",
    "    cm = np.vstack([cm_keep[:ind_toinsert, :], cm_ind, cm_keep[ind_toinsert:, :]])\n",
    "    # same for columns\n",
    "    cm_keep = cm[:, indices_keep]\n",
    "    cm_ind = cm[:, indices_merge]\n",
    "    cm_ind = np.sum(cm_ind, axis=1, keepdims=True)\n",
    "    cm = np.hstack([cm_keep[:, :ind_toinsert], cm_ind, cm_keep[:, ind_toinsert:]])\n",
    "    return cm\n",
    "\n",
    "def reduce_wake(cm):\n",
    "    # for confmat W, 1, 2, 3, R: if number of true wake epochs is higher than \n",
    "    # the biggest other class (always stage 2), limit the number of wake epochs to \n",
    "    # the number of S2 epochs\n",
    "    n_eps = np.sum(cm, axis=1)\n",
    "    n_wake = n_eps[0]\n",
    "    n_s2 = n_eps[2]\n",
    "    if n_wake > n_s2:\n",
    "        cm_wake = cm[0, :]\n",
    "        cm_wake = [np.round(n*n_s2/float(n_wake)) for n in cm_wake]\n",
    "        cm[0, :] = cm_wake\n",
    "    return cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_y_true_pred(cm):\n",
    "    # get lists y_true, y_pred from confmat\n",
    "    y_true = []\n",
    "    y_pred = []\n",
    "    for i, elt_i in enumerate(cm): #elmt_i is the line\n",
    "        for j, elt_j in enumerate(elt_i): #elmt_j is the value (number of occurences)\n",
    "            y_true += [i]*elt_j\n",
    "            y_pred += [j]*elt_j\n",
    "    return y_true, y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_metrics(y_true, y_pred):\n",
    "    acc = sklearn.metrics.accuracy_score(y_true, y_pred)\n",
    "    kappa = sklearn.metrics.cohen_kappa_score(y_true, y_pred)\n",
    "    f1_micro = sklearn.metrics.f1_score(y_true, y_pred, average='micro')\n",
    "    f1_macro = sklearn.metrics.f1_score(y_true, y_pred, average='macro')\n",
    "    sklearn_report = sklearn.metrics.classification_report(y_true, y_pred, target_names=['W', 'S1', 'S2', 'S3(&4)', 'R'])\n",
    "    return {\n",
    "        'accuracy': acc,\n",
    "        'kappa': kappa,\n",
    "        'f1_micro': f1_micro,\n",
    "        'f1_macro': f1_macro,\n",
    "        'sklearn report': sklearn_report,\n",
    "    }\n",
    "\n",
    "def add_infos(d, citation, dataset, nb_patients, scoring_rule, channel, model, split_type, cross_val, nb_raters_per_rec):\n",
    "    d_ = {\n",
    "        'dataset': dataset,\n",
    "        'patients': nb_patients,\n",
    "        'scoring rule': scoring_rule,\n",
    "        'channel': channel, \n",
    "        'model': model,\n",
    "        'split type': split_type,\n",
    "        'cross_val': cross_val,\n",
    "        'raters per record': nb_raters_per_rec,\n",
    "    }\n",
    "    # return {'article': '\\cite{%s}' %citation, **d_, **d} #python 3.5\n",
    "    return dict({'article': citation}.items() + d_.items() + d.items())\n",
    "\n",
    "def print_metrics(d, print_infos=True, print_metrics=True):\n",
    "    if print_infos:\n",
    "        print(\"Dataset: \", d['dataset'])\n",
    "        print(\"Number of patients: \", d['patients'])\n",
    "        print(\"Scoring rule: \", d['scoring rule'])\n",
    "        print(\"Channel: \", d['channel'])\n",
    "        print(\"Model: \", d['model'])\n",
    "        print(\"Split type: \", d['split type'])\n",
    "        print(\"Cross_val: \", d['cross_val'])\n",
    "        print(\"Number of raters per record: \", d['raters per record'])\n",
    "        print('\\n')\n",
    "    if print_metrics:\n",
    "        print(\"Accuracy: \", d['accuracy'])\n",
    "        print(\"Kappa: \", d['kappa'])\n",
    "        print('f1-micro: ', d['f1_micro'])\n",
    "        print('f1_macro: ', d['f1_macro'])\n",
    "        print('sklearn_report: ', '\\n', d['sklearn report'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tsinalis_2016\n",
    "Tsinalis, O., Matthews, P. M., Guo, Y., & Zafeiriou, S. (2016). Automatic sleep stage scoring with single-channel EEG using convolutional neural networks. arXiv preprint arXiv:1610.01683."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([[ 2382,   605,   171,    47,   175],\n",
      "       [  410,  1657,   259,     9,   427],\n",
      "       [  666,  1534, 12858,  1263,  1257],\n",
      "       [   85,     9,   399,  5097,     1],\n",
      "       [  360,  1019,   643,     3,  5686]]), '\\n')\n",
      "('Dataset: ', 'Sleep-EDF')\n",
      "('Number of patients: ', 20)\n",
      "('Scoring rule: ', 'R&K')\n",
      "('Channel: ', 'Fpz-Cz')\n",
      "('Model: ', 'CNN')\n",
      "('Split type: ', 'record')\n",
      "('Cross_val: ', '20-fold CV')\n",
      "('Number of raters per record: ', 1)\n",
      "\n",
      "\n",
      "('Accuracy: ', 0.74766355140186913)\n",
      "('Kappa: ', 0.65349732580287201)\n",
      "('f1-micro: ', 0.74766355140186913)\n",
      "('f1_macro: ', 0.69821619127089174)\n",
      "('sklearn_report: ', '\\n', '             precision    recall  f1-score   support\\n\\n          W       0.61      0.70      0.65      3380\\n         S1       0.34      0.60      0.44      2762\\n         S2       0.90      0.73      0.81     17578\\n     S3(&4)       0.79      0.91      0.85      5591\\n          R       0.75      0.74      0.75      7711\\n\\navg / total       0.78      0.75      0.76     37022\\n')\n"
     ]
    }
   ],
   "source": [
    "cm_tsinalis2016 = np.array([[1657, 259, 9, 427, 410],\n",
    "                            [1534, 12858, 1263, 1257, 666],\n",
    "                            [9, 399, 5097, 1, 85],\n",
    "                            [1019, 643, 3, 5686, 360],\n",
    "                            [605, 171, 47, 175, 2382]])\n",
    "# order: 1, 2, 3, R, W\n",
    "cm_tsinalis2016 = reorder(cm_tsinalis2016, [4, 0, 1, 2, 3])\n",
    "print(cm_tsinalis2016, '\\n')\n",
    "d_tsinalis_2016 = get_metrics(*get_y_true_pred(cm_tsinalis2016))\n",
    "d_tsinalis_2016 = add_infos(d_tsinalis_2016, 'tsinalis_2016', 'Sleep-EDF', 20, 'R&K', 'Fpz-Cz', 'CNN', 'record', '20-fold CV', 1)\n",
    "print_metrics(d_tsinalis_2016)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Supratak 2017\n",
    "\n",
    "SUPRATAK, Akara, DONG, Hao, WU, Chao, et al. DeepSleepNet: a Model for Automatic Sleep Stage Scoring based on Raw Single-Channel EEG. arXiv preprint arXiv:1703.04046, 2017.\n",
    "\n",
    "Rem: particularity: use of EOG:\n",
    "The channel used is F4-EOG: clever!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([[ 5433,   572,   107,    13,   102],\n",
      "       [  452,  2802,   827,     4,   639],\n",
      "       [  185,   906, 26786,  1158,   499],\n",
      "       [   18,     4,  1552,  6077,     0],\n",
      "       [  132,   356,   533,     1,  9442]]), '\\n')\n",
      "('Dataset: ', 'MASS')\n",
      "('Number of patients: ', 32)\n",
      "('Scoring rule: ', 'AASM')\n",
      "('Channel: ', 'F4-EOG')\n",
      "('Model: ', 'CNN-LSTM')\n",
      "('Split type: ', 'record')\n",
      "('Cross_val: ', '31-fold CV')\n",
      "('Number of raters per record: ', 1)\n",
      "\n",
      "\n",
      "('Accuracy: ', 0.86245733788395906)\n",
      "('Kappa: ', 0.79692134756215127)\n",
      "('f1-micro: ', 0.86245733788395906)\n",
      "('f1_macro: ', 0.81655429597108298)\n",
      "('sklearn_report: ', '\\n', '             precision    recall  f1-score   support\\n\\n          W       0.87      0.87      0.87      6227\\n         S1       0.60      0.59      0.60      4724\\n         S2       0.90      0.91      0.90     29534\\n     S3(&4)       0.84      0.79      0.82      7651\\n          R       0.88      0.90      0.89     10464\\n\\navg / total       0.86      0.86      0.86     58600\\n')\n"
     ]
    }
   ],
   "source": [
    "cm_supratak_2017 = np.array([[5433, 572, 107, 13, 102],\n",
    "                            [452, 2802, 827, 4, 639],\n",
    "                            [185, 906, 26786, 1158, 499],\n",
    "                            [18, 4, 1552, 6077, 0],\n",
    "                            [132, 356, 533, 1, 9442]])\n",
    "print(cm_supratak_2017, '\\n')\n",
    "d_supratak_2017 = get_metrics(*get_y_true_pred(cm_supratak_2017))\n",
    "d_supratak_2017 = add_infos(d_supratak_2017, 'supratak_2017', 'MASS', 32, 'AASM', 'F4-EOG', 'CNN-LSTM', 'record', '31-fold CV', 1)\n",
    "print_metrics(d_supratak_2017)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Liang 2012\n",
    "\n",
    "Liang, S. F., Kuo, C. E., Hu, Y. H., Pan, Y. H., & Wang, Y. H. (2012). Automatic stage scoring of single-channel sleep EEG by using multiscale entropy and autoregressive models. IEEE Transactions on Instrumentation and Measurement, 61(6), 1649-1657."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([[ 195,   24,    4,    0,    3],\n",
      "       [  31,   72,   48,    3,   69],\n",
      "       [  12,  103, 4078,  216,  220],\n",
      "       [   1,    4,  196, 1309,    0],\n",
      "       [   8,    8,   22,    6, 1818]]), '\\n')\n",
      "('Dataset: ', 'Custom')\n",
      "('Number of patients: ', 20)\n",
      "('Scoring rule: ', 'R&K')\n",
      "('Channel: ', 'C3-A2')\n",
      "('Model: ', 'Multiscale entropy, AR features, smoothing rules')\n",
      "('Split type: ', 'epoch')\n",
      "('Cross_val: ', '50/50')\n",
      "('Number of raters per record: ', 2)\n",
      "\n",
      "\n",
      "('Accuracy: ', 0.88426035502958578)\n",
      "('Kappa: ', 0.81605727956860696)\n",
      "('f1-micro: ', 0.88426035502958578)\n",
      "('f1_macro: ', 0.76806520447137139)\n",
      "('sklearn_report: ', '\\n', '             precision    recall  f1-score   support\\n\\n          W       0.79      0.86      0.82       226\\n         S1       0.34      0.32      0.33       223\\n         S2       0.94      0.88      0.91      4629\\n     S3(&4)       0.85      0.87      0.86      1510\\n          R       0.86      0.98      0.92      1862\\n\\navg / total       0.89      0.88      0.88      8450\\n')\n"
     ]
    }
   ],
   "source": [
    "cm_liang_2012 = np.array([[195, 24, 4, 0, 3],\n",
    "                     [31, 72, 48, 3, 69],\n",
    "                     [12, 103, 4078, 216, 220],\n",
    "                     [1, 4, 196, 1309, 0],\n",
    "                     [8, 8, 22, 6, 1818]])\n",
    "print(cm_liang_2012, '\\n')\n",
    "d_liang_2012 = get_metrics(*get_y_true_pred(cm_liang_2012))\n",
    "d_liang_2012 = add_infos(d_liang_2012, 'liang_2012', 'Custom', 20, 'R&K', 'C3-A2', 'Multiscale entropy, AR features, smoothing rules', 'epoch', '50/50', 2)\n",
    "print_metrics(d_liang_2012)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Zhu 2014\n",
    "\n",
    "Zhu, G., Li, Y., & Wen, P. P. (2014). Analysis and classification of sleep stages based on difference visibility graphs from a single-channel EEG signal. IEEE journal of biomedical and health informatics, 18(6), 1813-1821."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cm_zhu_2014 = np.array([[3863, 66, 23, 4, 20],\n",
    "                   [11, 50, 5, 3, 12],\n",
    "                   [5, 80, 1619, 139, 159],\n",
    "                   [1, 1, 74, 504, 0],\n",
    "                   [31, 120, 75, 2, 612]])\n",
    "cm_zhu_2014 = np.transpose(cm_zhu_2014)\n",
    "# REM: unusually low proportion of S1 !\n",
    "#print(cm_zhu_2014, '\\n')\n",
    "d_zhu_2014 = get_metrics(*get_y_true_pred(cm_zhu_2014))\n",
    "#print_metrics(d_zhu_2014)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([[1774,    5,    2,    0,   14],\n",
      "       [  66,   50,   80,    1,  120],\n",
      "       [  23,    5, 1619,   74,   75],\n",
      "       [   4,    3,  139,  504,    2],\n",
      "       [  20,   12,  159,    0,  612]]), '\\n')\n",
      "('Dataset: ', 'Sleep-EDF')\n",
      "('Number of patients: ', 20)\n",
      "('Scoring rule: ', 'R&K')\n",
      "('Channel: ', 'Pz-Oz')\n",
      "('Model: ', 'Difference visibility graph, SVM')\n",
      "('Split type: ', 'epoch')\n",
      "('Cross_val: ', '10-fold CV')\n",
      "('Number of raters per record: ', 1)\n",
      "\n",
      "\n",
      "('Accuracy: ', 0.85008390826030211)\n",
      "('Kappa: ', 0.7919216100492571)\n",
      "('f1-micro: ', 0.85008390826030211)\n",
      "('f1_macro: ', 0.72871014034756576)\n",
      "('sklearn_report: ', '\\n', '             precision    recall  f1-score   support\\n\\n          W       0.94      0.99      0.96      1795\\n         S1       0.67      0.16      0.26       317\\n         S2       0.81      0.90      0.85      1796\\n     S3(&4)       0.87      0.77      0.82       652\\n          R       0.74      0.76      0.75       803\\n\\navg / total       0.84      0.85      0.84      5363\\n')\n"
     ]
    }
   ],
   "source": [
    "cm_rw_zhu_2014 = reduce_wake(cm_zhu_2014)\n",
    "print(cm_rw_zhu_2014, '\\n')\n",
    "d_rw_zhu_2014 = get_metrics(*get_y_true_pred(cm_rw_zhu_2014))\n",
    "d_rw_zhu_2014 = add_infos(d_rw_zhu_2014, 'zhu_2014', 'Sleep-EDF', 20, 'R&K', 'Pz-Oz', 'Difference visibility graph, SVM', 'epoch', '10-fold CV', 1)\n",
    "print_metrics(d_rw_zhu_2014)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fraiwan 2013\n",
    "\n",
    "Fraiwan, L., Lweesy, K., Khasawneh, N., Wenz, H., & Dickhaus, H. (2012). Automated sleep stage identification system based on time–frequency analysis of a single EEG channel and random forest classifier. Computer methods and programs in biomedicine, 108(1), 10-19."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cm_fraiwan_2012 = np.array([[2407, 56, 69, 14, 33],\n",
    "                           [89, 185, 85, 9, 60],\n",
    "                           [11, 52, 1897, 86, 92],\n",
    "                           [38, 8, 174, 482, 3],\n",
    "                           [40, 48, 131, 3, 719]])\n",
    "cm_fraiwan_2012 = np.transpose(cm_fraiwan_2012)\n",
    "#print(cm_fraiwan_2012, '\\n')\n",
    "d_fraiwan_2012 = get_metrics(*get_y_true_pred(cm_fraiwan_2012))\n",
    "#print_metrics(d_fraiwan_2012)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([[1774,    5,    2,    0,   14],\n",
      "       [  66,   50,   80,    1,  120],\n",
      "       [  23,    5, 1619,   74,   75],\n",
      "       [   4,    3,  139,  504,    2],\n",
      "       [  20,   12,  159,    0,  612]]), '\\n')\n",
      "('Dataset: ', 'Custom')\n",
      "('Number of patients: ', 16)\n",
      "('Scoring rule: ', 'AASM')\n",
      "('Channel: ', 'C3-A1')\n",
      "('Model: ', 'Time-frequency feat., random forest')\n",
      "('Split type: ', 'epoch')\n",
      "('Cross_val: ', '67/33')\n",
      "('Number of raters per record: ', 3)\n",
      "\n",
      "\n",
      "('Accuracy: ', 0.83465406888143856)\n",
      "('Kappa: ', 0.77060637491945616)\n",
      "('f1-micro: ', 0.83465406888143856)\n",
      "('f1_macro: ', 0.75569797764478752)\n",
      "('sklearn_report: ', '\\n', '             precision    recall  f1-score   support\\n\\n          W       0.93      0.93      0.93      2356\\n         S1       0.44      0.53      0.48       349\\n         S2       0.89      0.81      0.84      2356\\n     S3(&4)       0.69      0.81      0.74       594\\n          R       0.77      0.79      0.78       907\\n\\navg / total       0.84      0.83      0.84      6562\\n')\n"
     ]
    }
   ],
   "source": [
    "cm_rw_fraiwan_2012 = reduce_wake(cm_fraiwan_2012)\n",
    "print(cm_rw_zhu_2014, '\\n')\n",
    "d_rw_fraiwan_2012 = get_metrics(*get_y_true_pred(cm_rw_fraiwan_2012))\n",
    "d_rw_fraiwan_2012 = add_infos(d_rw_fraiwan_2012, 'fraiwan_2012', 'Custom', 16, 'AASM', 'C3-A1', 'Time-frequency feat., random forest', 'epoch', '67/33', 3)\n",
    "print_metrics(d_rw_fraiwan_2012)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hassan 2016\n",
    "\n",
    "Hassan, A. R., & Bhuiyan, M. I. H. (2016). Automatic sleep scoring using statistical features in the EMD domain and ensemble methods. Biocybernetics and Biomedical Engineering, 36(1), 248-255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cm_hassan_2016 = np.array([[120, 60, 1, 33, 88],\n",
    "                        [4, 1622, 61, 75, 49],\n",
    "                        [2, 93, 519, 35, 1],\n",
    "                        [18, 35, 3, 3945, 16],\n",
    "                        [19, 130, 1, 26, 629]])\n",
    "#order: 1, 2, 3, W, R\n",
    "cm_hassan_2016 = reorder(cm_hassan_2016, [3, 0, 1, 2, 4])\n",
    "#print(cm_hassan_2016, '\\n')\n",
    "d_hassan_2016 = get_metrics(*get_y_true_pred(cm_hassan_2016))\n",
    "#print_metrics(d_hassan_2016)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([[1779,    8,   16,    1,    7],\n",
      "       [  33,  120,   60,    1,   88],\n",
      "       [  75,    4, 1622,   61,   49],\n",
      "       [  35,    2,   93,  519,    1],\n",
      "       [  26,   19,  130,    1,  629]]), '\\n')\n",
      "('Dataset: ', 'Sleep-EDF')\n",
      "('Number of patients: ', 20)\n",
      "('Scoring rule: ', 'R&K')\n",
      "('Channel: ', 'Pz-Oz')\n",
      "('Model: ', 'EMD domain, ensemble')\n",
      "('Split type: ', 'epoch')\n",
      "('Cross_val: ', '0.6/0.05/0.35')\n",
      "('Number of raters per record: ', 1)\n",
      "\n",
      "\n",
      "('Accuracy: ', 0.86800520542851833)\n",
      "('Kappa: ', 0.8170816167477446)\n",
      "('f1-micro: ', 0.86800520542851844)\n",
      "('f1_macro: ', 0.79635915965370885)\n",
      "('sklearn_report: ', '\\n', '             precision    recall  f1-score   support\\n\\n          W       0.91      0.98      0.95      1811\\n         S1       0.78      0.40      0.53       302\\n         S2       0.84      0.90      0.87      1811\\n     S3(&4)       0.89      0.80      0.84       650\\n          R       0.81      0.78      0.80       805\\n\\navg / total       0.86      0.87      0.86      5379\\n')\n"
     ]
    }
   ],
   "source": [
    "cm_rw_hassan_2016 = reduce_wake(cm_hassan_2016)\n",
    "print(cm_rw_hassan_2016, '\\n')\n",
    "d_rw_hassan_2016 = get_metrics(*get_y_true_pred(cm_rw_hassan_2016))\n",
    "d_rw_hassan_2016 = add_infos(d_rw_hassan_2016, 'hassan_2016', 'Sleep-EDF', 20, 'R&K', 'Pz-Oz', 'EMD domain, ensemble', 'epoch', '0.6/0.05/0.35', 1)\n",
    "print_metrics(d_rw_hassan_2016)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hassan 2016_2\n",
    "\n",
    "Hassan, A. R., & Bhuiyan, M. I. H. (2016). Computer-aided sleep staging using complete ensemble empirical mode decomposition with adaptive noise and bootstrap aggregating. Biomedical Signal Processing and Control, 24, 1-10."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cm_hassan_2016_2 = np.array([[142, 55, 1, 32, 72],\n",
    "                          [3, 1673, 45, 54, 36],\n",
    "                          [4, 43, 585, 16, 2],\n",
    "                          [42, 79, 18, 3838, 51],\n",
    "                          [17, 109, 4, 24, 651]])\n",
    "cm_hassan_2016_2 = reorder(cm_hassan_2016_2, [3, 0, 1, 2, 4])\n",
    "#print(cm_hassan_2016_2, '\\n')\n",
    "d_hassan_2016_2 = get_metrics(*get_y_true_pred(cm_hassan_2016_2))\n",
    "#print_metrics(d_hassan_2016_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([[1726,   19,   36,    8,   23],\n",
      "       [  32,  142,   55,    1,   72],\n",
      "       [  54,    3, 1673,   45,   36],\n",
      "       [  16,    4,   43,  585,    2],\n",
      "       [  24,   17,  109,    4,  651]]), '\\n')\n",
      "('Dataset: ', 'Sleep-EDF')\n",
      "('Number of patients: ', 20)\n",
      "('Scoring rule: ', 'R&K')\n",
      "('Channel: ', 'Pz-Oz')\n",
      "('Model: ', 'EMD, bootstrap aggregating')\n",
      "('Split type: ', 'epoch')\n",
      "('Cross_val: ', '50/50')\n",
      "('Number of raters per record: ', 1)\n",
      "\n",
      "\n",
      "('Accuracy: ', 0.88791821561338291)\n",
      "('Kappa: ', 0.84561829962843393)\n",
      "('f1-micro: ', 0.8879182156133828)\n",
      "('f1_macro: ', 0.82946612262690778)\n",
      "('sklearn_report: ', '\\n', '             precision    recall  f1-score   support\\n\\n          W       0.93      0.95      0.94      1812\\n         S1       0.77      0.47      0.58       302\\n         S2       0.87      0.92      0.90      1811\\n     S3(&4)       0.91      0.90      0.90       650\\n          R       0.83      0.81      0.82       805\\n\\navg / total       0.89      0.89      0.88      5380\\n')\n"
     ]
    }
   ],
   "source": [
    "cm_rw_hassan_2016_2 = reduce_wake(cm_hassan_2016_2)\n",
    "print(cm_rw_hassan_2016_2, '\\n')\n",
    "d_rw_hassan_2016_2 = get_metrics(*get_y_true_pred(cm_rw_hassan_2016_2))\n",
    "d_rw_hassan_2016_2 = add_infos(d_rw_hassan_2016_2, 'hassan_2016_2', 'Sleep-EDF', 20, 'R&K', 'Pz-Oz', 'EMD, bootstrap aggregating', 'epoch', '50/50', 1)\n",
    "print_metrics(d_rw_hassan_2016_2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hassan_2016_3\n",
    "\n",
    "Hassan, A. R., & Bhuiyan, M. I. H. (2016). A decision support system for automatic sleep staging from EEG signals using tunable Q-factor wavelet transform and spectral features. Journal of neuroscience methods, 271, 107-118."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cm_hassan_2016_3 = np.array([[113, 41, 1, 74, 73],\n",
    "                          [7, 1656, 59, 49, 40],\n",
    "                          [0, 96, 527, 27, 0],\n",
    "                          [24, 5, 3, 3945, 21],\n",
    "                          [33, 70, 0, 41, 661]])\n",
    "cm_hassan_2016_3 = reorder(cm_hassan_2016_3, [3, 0, 1, 2, 4])\n",
    "#print(cm_hassan_2016_3, '\\n')\n",
    "d_hassan_2016_3 = get_metrics(*get_y_true_pred(cm_hassan_2016_3))\n",
    "#print_metrics(d_hassan_2016_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([[1787,   11,    2,    1,   10],\n",
      "       [  74,  113,   41,    1,   73],\n",
      "       [  49,    7, 1656,   59,   40],\n",
      "       [  27,    0,   96,  527,    0],\n",
      "       [  41,   33,   70,    0,  661]]), '\\n')\n",
      "('Dataset: ', 'Sleep-EDF')\n",
      "('Number of patients: ', 20)\n",
      "('Scoring rule: ', 'R&K')\n",
      "('Channel: ', 'Pz-Oz')\n",
      "('Model: ', 'Wavelet transform, spectral features, random forest')\n",
      "('Split type: ', 'epoch')\n",
      "('Cross_val: ', '50/50, 20-fold average')\n",
      "('Number of raters per record: ', 1)\n",
      "\n",
      "\n",
      "('Accuracy: ', 0.88194831753113967)\n",
      "('Kappa: ', 0.8366579599753623)\n",
      "('f1-micro: ', 0.88194831753113967)\n",
      "('f1_macro: ', 0.80251152739785669)\n",
      "('sklearn_report: ', '\\n', '             precision    recall  f1-score   support\\n\\n          W       0.90      0.99      0.94      1811\\n         S1       0.69      0.37      0.48       302\\n         S2       0.89      0.91      0.90      1811\\n     S3(&4)       0.90      0.81      0.85       650\\n          R       0.84      0.82      0.83       805\\n\\navg / total       0.88      0.88      0.88      5379\\n')\n"
     ]
    }
   ],
   "source": [
    "cm_rw_hassan_2016_3 = reduce_wake(cm_hassan_2016_3)\n",
    "print(cm_rw_hassan_2016_3, '\\n')\n",
    "d_rw_hassan_2016_3 = get_metrics(*get_y_true_pred(cm_rw_hassan_2016_3))\n",
    "d_rw_hassan_2016_3 = add_infos(d_rw_hassan_2016_3, 'hassan_2016_3', 'Sleep-EDF', 20, 'R&K', 'Pz-Oz', 'Wavelet transform, spectral features, random forest', 'epoch', '50/50, 20-fold average', 1)\n",
    "print_metrics(d_rw_hassan_2016_3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hassan 2017\n",
    "\n",
    "Hassan, A. R., & Bhuiyan, M. I. H. (2017). Automated identification of sleep states from EEG signals by means of ensemble empirical mode decomposition and random under sampling boosting. Computer Methods and Programs in Biomedicine, 140, 201-210."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cm_hassan_2017 = np.array([[127, 16, 0, 0, 17, 142],\n",
    "                        [65, 1440, 89, 36, 34, 147],\n",
    "                        [0, 4, 291, 32, 9, 0],\n",
    "                        [37, 80, 29, 151, 17, 0],\n",
    "                        [37, 52, 7, 16, 3833, 83],\n",
    "                        [10, 68, 0, 32, 47, 648]])\n",
    "cm_hassan_2017 = merge(cm_hassan_2017)\n",
    "cm_hassan_2017 = reorder(cm_hassan_2017, [3, 0, 1, 2, 4])\n",
    "#print(cm_hassan_2017, '\\n')\n",
    "d_hassan_2017 = get_metrics(*get_y_true_pred(cm_hassan_2017))\n",
    "#print_metrics(d_hassan_2017)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([[1723,   17,   23,   10,   37],\n",
      "       [  17,  127,   16,    0,  142],\n",
      "       [  34,   65, 1440,  125,  147],\n",
      "       [  26,   37,   84,  503,    0],\n",
      "       [  47,   10,   68,   32,  648]]), '\\n')\n",
      "('Dataset: ', 'Sleep-EDF')\n",
      "('Number of patients: ', 20)\n",
      "('Scoring rule: ', 'R&K')\n",
      "('Channel: ', 'Pz-Oz')\n",
      "('Model: ', 'EMD, random undersampling boosting')\n",
      "('Split type: ', 'epoch')\n",
      "('Cross_val: ', '50/50, 20-fold average')\n",
      "('Number of raters per record: ', 1)\n",
      "\n",
      "\n",
      "('Accuracy: ', 0.82577166232800292)\n",
      "('Kappa: ', 0.7637422816458439)\n",
      "('f1-micro: ', 0.82577166232800292)\n",
      "('f1_macro: ', 0.74496855416338315)\n",
      "('sklearn_report: ', '\\n', '             precision    recall  f1-score   support\\n\\n          W       0.93      0.95      0.94      1810\\n         S1       0.50      0.42      0.46       302\\n         S2       0.88      0.80      0.84      1811\\n     S3(&4)       0.75      0.77      0.76       650\\n          R       0.67      0.80      0.73       805\\n\\navg / total       0.83      0.83      0.83      5378\\n')\n"
     ]
    }
   ],
   "source": [
    "cm_rw_hassan_2017 = reduce_wake(cm_hassan_2017)\n",
    "print(cm_rw_hassan_2017, '\\n')\n",
    "d_rw_hassan_2017 = get_metrics(*get_y_true_pred(cm_rw_hassan_2017))\n",
    "d_rw_hassan_2017 = add_infos(d_rw_hassan_2017, 'hassan_2017', 'Sleep-EDF', 20, 'R&K', 'Pz-Oz', 'EMD, random undersampling boosting', 'epoch', '50/50, 20-fold average', 1)\n",
    "print_metrics(d_rw_hassan_2017)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sharma 2017\n",
    "\n",
    "Sharma, R., Pachori, R. B., & Upadhyay, A. (2017). Automatic sleep stages classification based on iterative filtering of electroencephalogram signals. Neural Computing and Applications, 1-20."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cm_sharma_2017 = np.array([[7944, 11, 12, 6, 30],\n",
    "                        [183, 113, 123, 4, 181],\n",
    "                        [48, 4, 3334, 149, 86],\n",
    "                        [13, 0, 198, 1088, 0],\n",
    "                        [51, 11, 207, 0, 1339],])\n",
    "#print(cm_sharma_2017, '\\n')\n",
    "d_sharma_2017 = get_metrics(*get_y_true_pred(cm_sharma_2017))\n",
    "#print_metrics(d_sharma_2017)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([[3594,    5,    5,    3,   14],\n",
      "       [ 183,  113,  123,    4,  181],\n",
      "       [  48,    4, 3334,  149,   86],\n",
      "       [  13,    0,  198, 1088,    0],\n",
      "       [  51,   11,  207,    0, 1339]]), '\\n')\n",
      "('Dataset: ', 'Sleep-EDF')\n",
      "('Number of patients: ', 20)\n",
      "('Scoring rule: ', 'R&K')\n",
      "('Channel: ', 'Pz-Oz')\n",
      "('Model: ', 'Iterative filtering')\n",
      "('Split type: ', 'epoch')\n",
      "('Cross_val: ', '10-fold CV')\n",
      "('Number of raters per record: ', 1)\n",
      "\n",
      "\n",
      "('Accuracy: ', 0.88049846554449918)\n",
      "('Kappa: ', 0.83400538566890225)\n",
      "('f1-micro: ', 0.88049846554449918)\n",
      "('f1_macro: ', 0.76791239788784527)\n",
      "('sklearn_report: ', '\\n', '             precision    recall  f1-score   support\\n\\n          W       0.92      0.99      0.96      3621\\n         S1       0.85      0.19      0.31       604\\n         S2       0.86      0.92      0.89      3621\\n     S3(&4)       0.87      0.84      0.86      1299\\n          R       0.83      0.83      0.83      1608\\n\\navg / total       0.88      0.88      0.87     10753\\n')\n"
     ]
    }
   ],
   "source": [
    "cm_rw_sharma_2017 = reduce_wake(cm_sharma_2017)\n",
    "print(cm_rw_sharma_2017, '\\n')\n",
    "d_rw_sharma_2017 = get_metrics(*get_y_true_pred(cm_rw_sharma_2017))\n",
    "d_rw_sharma_2017 = add_infos(d_rw_sharma_2017, 'sharma_2017', 'Sleep-EDF', 20, 'R&K', 'Pz-Oz', 'Iterative filtering', 'epoch', '10-fold CV', 1)\n",
    "print_metrics(d_rw_sharma_2017)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hsu 2013\n",
    "\n",
    "Hsu, Y. L., Yang, Y. T., Wang, J. S., & Hsu, C. Y. (2013). Automatic sleep stage recurrent neural classifier using energy features of EEG signals. Neurocomputing, 104, 105-114."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([[ 34,   2,   7,   2,   3],\n",
      "       [  0,  20,  23,   3,   9],\n",
      "       [  3,   4, 574,   8,   1],\n",
      "       [  0,   0,   3,  26,   0],\n",
      "       [  3,   5,  13,   4, 213]]), '\\n')\n",
      "('Dataset: ', 'Sleep-EDF')\n",
      "('Number of patients: ', 8)\n",
      "('Scoring rule: ', 'R&K')\n",
      "('Channel: ', 'Fpz-Cz')\n",
      "('Model: ', 'Energy features, recurrent neural classifier')\n",
      "('Split type: ', 'epoch')\n",
      "('Cross_val: ', '10-fold CV')\n",
      "('Number of raters per record: ', 1)\n",
      "\n",
      "\n",
      "('Accuracy: ', 0.90312499999999996)\n",
      "('Kappa: ', 0.82041276099287908)\n",
      "('f1-micro: ', 0.90312499999999996)\n",
      "('f1_macro: ', 0.76538591057472727)\n",
      "('sklearn_report: ', '\\n', '             precision    recall  f1-score   support\\n\\n          W       0.85      0.71      0.77        48\\n         S1       0.65      0.36      0.47        55\\n         S2       0.93      0.97      0.95       590\\n     S3(&4)       0.60      0.90      0.72        29\\n          R       0.94      0.89      0.92       238\\n\\navg / total       0.90      0.90      0.90       960\\n')\n"
     ]
    }
   ],
   "source": [
    "cm_hsu_2013 = np.array([[20, 23, 3, 9, 0],\n",
    "                     [4, 574, 8, 1, 3],\n",
    "                     [0, 3, 26, 0, 0],\n",
    "                     [5, 13, 4, 213, 3],\n",
    "                     [2, 7, 2, 3, 34]])\n",
    "cm_hsu_2013 = reorder(cm_hsu_2013, [4, 0, 1, 2, 3])\n",
    "print(cm_hsu_2013, '\\n')\n",
    "d_hsu_2013 = get_metrics(*get_y_true_pred(cm_hsu_2013))\n",
    "d_hsu_2013 = add_infos(d_hsu_2013, 'hsu_2013', 'Sleep-EDF', 8, 'R&K', 'Fpz-Cz', 'Energy features, recurrent neural classifier', 'epoch', '10-fold CV', 1)\n",
    "print_metrics(d_hsu_2013)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cm_ours_normalized: \n",
      "[[ 0.91  0.02  0.04  0.01  0.02]\n",
      " [ 0.19  0.35  0.31  0.    0.15]\n",
      " [ 0.03  0.01  0.89  0.04  0.03]\n",
      " [ 0.01  0.    0.14  0.85  0.  ]\n",
      " [ 0.04  0.01  0.09  0.    0.86]]\n",
      "('Dataset: ', 'SHHS-1')\n",
      "('Number of patients: ', 5728)\n",
      "('Scoring rule: ', 'R&K')\n",
      "('Channel: ', 'C4-A1')\n",
      "('Model: ', 'CNN')\n",
      "('Split type: ', 'record')\n",
      "('Cross_val: ', '0.5/0.2/0.3')\n",
      "('Number of raters per record: ', 1)\n",
      "\n",
      "\n",
      "('Accuracy: ', 0.86791448962968853)\n",
      "('Kappa: ', 0.81483393589670505)\n",
      "('f1-micro: ', 0.86791448962968853)\n",
      "('f1_macro: ', 0.78487205539183036)\n",
      "('sklearn_report: ', '\\n', '             precision    recall  f1-score   support\\n\\n          W       0.91      0.91      0.91    447335\\n         S1       0.55      0.35      0.43     59029\\n         S2       0.87      0.89      0.88    653195\\n     S3(&4)       0.85      0.85      0.85    215840\\n          R       0.85      0.86      0.85    235353\\n\\navg / total       0.86      0.87      0.86   1610752\\n')\n"
     ]
    }
   ],
   "source": [
    "cm_ours_old_old = np.array([[212900, 2704, 9679, 853, 3830], \n",
    "                   [7306, 11044, 7470, 11, 5650],\n",
    "                   [7996, 4308, 300692, 14691, 11442],\n",
    "                   [983, 2, 15259, 94259, 70],\n",
    "                   [6858, 1979, 8364, 41, 103993]])\n",
    "\n",
    "cm_ours_old = np.array([[269094,   4122,  12261,   1273,   3576],\n",
    "       [  7812,  14015,  11533,     42,   5588],\n",
    "       [ 10595,   4161, 380571,  16394,  10085],\n",
    "       [   847,      2,  21560, 119745,    171],\n",
    "       [  6545,   2558,  15748,    156, 128586]])\n",
    "\n",
    "cm_ours = np.array([[408881, 7010, 19800, 3740, 7904],\n",
    "                        [11161, 20614, 18152, 55, 9047],\n",
    "                        [16547, 6425, 582886, 28899, 18438],\n",
    "                        [1816, 7, 30050, 183631, 336],\n",
    "                        [8793, 3477, 20914, 186, 201983]])\n",
    "cm_ours_normalized = cm_ours.astype(np.float) / np.sum(cm_ours, axis=1)[:, np.newaxis]\n",
    "print('cm_ours_normalized: ')\n",
    "print(np.around(cm_ours_normalized, decimals=2))\n",
    "#print(cm_ours, '\\n')\n",
    "d_ours = get_metrics(*get_y_true_pred(cm_ours))\n",
    "d_ours = add_infos(d_ours, 'ours', 'SHHS-1', 5728, 'R&K', 'C4-A1', 'CNN', 'record', '0.5/0.2/0.3', 1)\n",
    "print_metrics(d_ours)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          W       0.91      0.91      0.91    447335\n",
      "         S1       0.55      0.35      0.43     59029\n",
      "         S2       0.87      0.89      0.88    653195\n",
      "     S3(&4)       0.85      0.85      0.85    215840\n",
      "          R       0.85      0.86      0.85    235353\n",
      "\n",
      "avg / total       0.86      0.87      0.86   1610752\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print '             precision    recall  f1-score   support\\n\\n          W       0.91      0.91      0.91    447335\\n         S1       0.55      0.35      0.43     59029\\n         S2       0.87      0.89      0.88    653195\\n     S3(&4)       0.85      0.85      0.85    215840\\n          R       0.85      0.86      0.85    235353\\n\\navg / total       0.86      0.87      0.86   1610752\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['split type',\n",
       " 'kappa',\n",
       " 'scoring rule',\n",
       " 'f1_macro',\n",
       " 'dataset',\n",
       " 'sklearn report',\n",
       " 'patients',\n",
       " 'f1_micro',\n",
       " 'raters per record',\n",
       " 'article',\n",
       " 'model',\n",
       " 'cross_val',\n",
       " 'channel',\n",
       " 'accuracy']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d_ours.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "col = ['article', 'dataset', 'channel', 'patients', 'raters per record', 'scoring rule', 'model', 'split type', 'cross_val', 'accuracy', 'kappa', 'f1_micro', 'f1_macro']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article</th>\n",
       "      <th>dataset</th>\n",
       "      <th>channel</th>\n",
       "      <th>patients</th>\n",
       "      <th>raters per record</th>\n",
       "      <th>scoring rule</th>\n",
       "      <th>model</th>\n",
       "      <th>split type</th>\n",
       "      <th>cross_val</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>kappa</th>\n",
       "      <th>f1_micro</th>\n",
       "      <th>f1_macro</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ours</td>\n",
       "      <td>SHHS-1</td>\n",
       "      <td>C4-A1</td>\n",
       "      <td>5728</td>\n",
       "      <td>1</td>\n",
       "      <td>R&amp;K</td>\n",
       "      <td>CNN</td>\n",
       "      <td>record</td>\n",
       "      <td>0.5/0.2/0.3</td>\n",
       "      <td>0.867914</td>\n",
       "      <td>0.814834</td>\n",
       "      <td>0.867914</td>\n",
       "      <td>0.784872</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  article dataset channel  patients  raters per record scoring rule model  \\\n",
       "0    ours  SHHS-1   C4-A1      5728                  1          R&K   CNN   \n",
       "\n",
       "  split type    cross_val  accuracy     kappa  f1_micro  f1_macro  \n",
       "0     record  0.5/0.2/0.3  0.867914  0.814834  0.867914  0.784872  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ours = pd.DataFrame([d_ours], columns=col)#, columns=d_ours.keys())\n",
    "df_ours"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Assemble results in a pd.DataFrame\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "studies_results = [d_tsinalis_2016, d_supratak_2017, d_liang_2012, d_rw_zhu_2014, d_rw_fraiwan_2012, d_rw_hassan_2016, \n",
    "                   d_rw_hassan_2016_2, d_rw_hassan_2016_3, d_rw_hassan_2017, d_rw_sharma_2017, d_hsu_2013, d_ours]\n",
    "\n",
    "df = pd.DataFrame([d_tsinalis_2016], columns=col).round(2)\n",
    "for r in range(1, len(studies_results)):\n",
    "    df_ = pd.DataFrame([studies_results[r]], columns=col).round(2)\n",
    "    df = pd.concat([df, df_])\n",
    "#del df['sklearn report']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article</th>\n",
       "      <th>dataset</th>\n",
       "      <th>channel</th>\n",
       "      <th>patients</th>\n",
       "      <th>raters per record</th>\n",
       "      <th>scoring rule</th>\n",
       "      <th>model</th>\n",
       "      <th>split type</th>\n",
       "      <th>cross_val</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>kappa</th>\n",
       "      <th>f1_micro</th>\n",
       "      <th>f1_macro</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tsinalis_2016</td>\n",
       "      <td>Sleep-EDF</td>\n",
       "      <td>Fpz-Cz</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>R&amp;K</td>\n",
       "      <td>CNN</td>\n",
       "      <td>record</td>\n",
       "      <td>20-fold CV</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.65</td>\n",
       "      <td>0.75</td>\n",
       "      <td>0.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>supratak_2017</td>\n",
       "      <td>MASS</td>\n",
       "      <td>F4-EOG</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>AASM</td>\n",
       "      <td>CNN-LSTM</td>\n",
       "      <td>record</td>\n",
       "      <td>31-fold CV</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.80</td>\n",
       "      <td>0.86</td>\n",
       "      <td>0.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>liang_2012</td>\n",
       "      <td>Custom</td>\n",
       "      <td>C3-A2</td>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>R&amp;K</td>\n",
       "      <td>Multiscale entropy, AR features, smoothing rules</td>\n",
       "      <td>epoch</td>\n",
       "      <td>50/50</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>zhu_2014</td>\n",
       "      <td>Sleep-EDF</td>\n",
       "      <td>Pz-Oz</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>R&amp;K</td>\n",
       "      <td>Difference visibility graph, SVM</td>\n",
       "      <td>epoch</td>\n",
       "      <td>10-fold CV</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.79</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fraiwan_2012</td>\n",
       "      <td>Custom</td>\n",
       "      <td>C3-A1</td>\n",
       "      <td>16</td>\n",
       "      <td>3</td>\n",
       "      <td>AASM</td>\n",
       "      <td>Time-frequency feat., random forest</td>\n",
       "      <td>epoch</td>\n",
       "      <td>67/33</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.77</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>hassan_2016</td>\n",
       "      <td>Sleep-EDF</td>\n",
       "      <td>Pz-Oz</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>R&amp;K</td>\n",
       "      <td>EMD domain, ensemble</td>\n",
       "      <td>epoch</td>\n",
       "      <td>0.6/0.05/0.35</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>hassan_2016_2</td>\n",
       "      <td>Sleep-EDF</td>\n",
       "      <td>Pz-Oz</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>R&amp;K</td>\n",
       "      <td>EMD, bootstrap aggregating</td>\n",
       "      <td>epoch</td>\n",
       "      <td>50/50</td>\n",
       "      <td>0.89</td>\n",
       "      <td>0.85</td>\n",
       "      <td>0.89</td>\n",
       "      <td>0.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>hassan_2016_3</td>\n",
       "      <td>Sleep-EDF</td>\n",
       "      <td>Pz-Oz</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>R&amp;K</td>\n",
       "      <td>Wavelet transform, spectral features, random f...</td>\n",
       "      <td>epoch</td>\n",
       "      <td>50/50, 20-fold average</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.84</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>hassan_2017</td>\n",
       "      <td>Sleep-EDF</td>\n",
       "      <td>Pz-Oz</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>R&amp;K</td>\n",
       "      <td>EMD, random undersampling boosting</td>\n",
       "      <td>epoch</td>\n",
       "      <td>50/50, 20-fold average</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sharma_2017</td>\n",
       "      <td>Sleep-EDF</td>\n",
       "      <td>Pz-Oz</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>R&amp;K</td>\n",
       "      <td>Iterative filtering</td>\n",
       "      <td>epoch</td>\n",
       "      <td>10-fold CV</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.83</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>hsu_2013</td>\n",
       "      <td>Sleep-EDF</td>\n",
       "      <td>Fpz-Cz</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>R&amp;K</td>\n",
       "      <td>Energy features, recurrent neural classifier</td>\n",
       "      <td>epoch</td>\n",
       "      <td>10-fold CV</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.90</td>\n",
       "      <td>0.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ours</td>\n",
       "      <td>SHHS-1</td>\n",
       "      <td>C4-A1</td>\n",
       "      <td>5728</td>\n",
       "      <td>1</td>\n",
       "      <td>R&amp;K</td>\n",
       "      <td>CNN</td>\n",
       "      <td>record</td>\n",
       "      <td>0.5/0.2/0.3</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.81</td>\n",
       "      <td>0.87</td>\n",
       "      <td>0.78</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         article    dataset channel  patients  raters per record scoring rule  \\\n",
       "0  tsinalis_2016  Sleep-EDF  Fpz-Cz        20                  1          R&K   \n",
       "0  supratak_2017       MASS  F4-EOG        32                  1         AASM   \n",
       "0     liang_2012     Custom   C3-A2        20                  2          R&K   \n",
       "0       zhu_2014  Sleep-EDF   Pz-Oz        20                  1          R&K   \n",
       "0   fraiwan_2012     Custom   C3-A1        16                  3         AASM   \n",
       "0    hassan_2016  Sleep-EDF   Pz-Oz        20                  1          R&K   \n",
       "0  hassan_2016_2  Sleep-EDF   Pz-Oz        20                  1          R&K   \n",
       "0  hassan_2016_3  Sleep-EDF   Pz-Oz        20                  1          R&K   \n",
       "0    hassan_2017  Sleep-EDF   Pz-Oz        20                  1          R&K   \n",
       "0    sharma_2017  Sleep-EDF   Pz-Oz        20                  1          R&K   \n",
       "0       hsu_2013  Sleep-EDF  Fpz-Cz         8                  1          R&K   \n",
       "0           ours     SHHS-1   C4-A1      5728                  1          R&K   \n",
       "\n",
       "                                               model split type  \\\n",
       "0                                                CNN     record   \n",
       "0                                           CNN-LSTM     record   \n",
       "0   Multiscale entropy, AR features, smoothing rules      epoch   \n",
       "0                   Difference visibility graph, SVM      epoch   \n",
       "0                Time-frequency feat., random forest      epoch   \n",
       "0                               EMD domain, ensemble      epoch   \n",
       "0                         EMD, bootstrap aggregating      epoch   \n",
       "0  Wavelet transform, spectral features, random f...      epoch   \n",
       "0                 EMD, random undersampling boosting      epoch   \n",
       "0                                Iterative filtering      epoch   \n",
       "0       Energy features, recurrent neural classifier      epoch   \n",
       "0                                                CNN     record   \n",
       "\n",
       "                cross_val  accuracy  kappa  f1_micro  f1_macro  \n",
       "0              20-fold CV      0.75   0.65      0.75      0.70  \n",
       "0              31-fold CV      0.86   0.80      0.86      0.82  \n",
       "0                   50/50      0.88   0.82      0.88      0.77  \n",
       "0              10-fold CV      0.85   0.79      0.85      0.73  \n",
       "0                   67/33      0.83   0.77      0.83      0.76  \n",
       "0           0.6/0.05/0.35      0.87   0.82      0.87      0.80  \n",
       "0                   50/50      0.89   0.85      0.89      0.83  \n",
       "0  50/50, 20-fold average      0.88   0.84      0.88      0.80  \n",
       "0  50/50, 20-fold average      0.83   0.76      0.83      0.74  \n",
       "0              10-fold CV      0.88   0.83      0.88      0.77  \n",
       "0              10-fold CV      0.90   0.82      0.90      0.77  \n",
       "0             0.5/0.2/0.3      0.87   0.81      0.87      0.78  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lllrrllllrrrr}\n",
      "\\toprule\n",
      "       article &    dataset & channel &  patients &  raters per record & scoring rule &                                              model & split type &               cross\\_val &  accuracy &  kappa &  f1\\_micro &  f1\\_macro \\\\\n",
      "\\midrule\n",
      " tsinalis\\_2016 &  Sleep-EDF &  Fpz-Cz &        20 &                  1 &          R\\&K &                                                CNN &     record &              20-fold CV &      0.75 &   0.65 &      0.75 &      0.70 \\\\\n",
      " supratak\\_2017 &       MASS &  F4-EOG &        32 &                  1 &         AASM &                                           CNN-LSTM &     record &              31-fold CV &      0.86 &   0.80 &      0.86 &      0.82 \\\\\n",
      "    liang\\_2012 &     Custom &   C3-A2 &        20 &                  2 &          R\\&K &   Multiscale entropy, AR features, smoothing rules &      epoch &                   50/50 &      0.88 &   0.82 &      0.88 &      0.77 \\\\\n",
      "      zhu\\_2014 &  Sleep-EDF &   Pz-Oz &        20 &                  1 &          R\\&K &                   Difference visibility graph, SVM &      epoch &              10-fold CV &      0.85 &   0.79 &      0.85 &      0.73 \\\\\n",
      "  fraiwan\\_2012 &     Custom &   C3-A1 &        16 &                  3 &         AASM &                Time-frequency feat., random forest &      epoch &                   67/33 &      0.83 &   0.77 &      0.83 &      0.76 \\\\\n",
      "   hassan\\_2016 &  Sleep-EDF &   Pz-Oz &        20 &                  1 &          R\\&K &                               EMD domain, ensemble &      epoch &           0.6/0.05/0.35 &      0.87 &   0.82 &      0.87 &      0.80 \\\\\n",
      " hassan\\_2016\\_2 &  Sleep-EDF &   Pz-Oz &        20 &                  1 &          R\\&K &                         EMD, bootstrap aggregating &      epoch &                   50/50 &      0.89 &   0.85 &      0.89 &      0.83 \\\\\n",
      " hassan\\_2016\\_3 &  Sleep-EDF &   Pz-Oz &        20 &                  1 &          R\\&K &  Wavelet transform, spectral features, random f... &      epoch &  50/50, 20-fold average &      0.88 &   0.84 &      0.88 &      0.80 \\\\\n",
      "   hassan\\_2017 &  Sleep-EDF &   Pz-Oz &        20 &                  1 &          R\\&K &                 EMD, random undersampling boosting &      epoch &  50/50, 20-fold average &      0.83 &   0.76 &      0.83 &      0.74 \\\\\n",
      "   sharma\\_2017 &  Sleep-EDF &   Pz-Oz &        20 &                  1 &          R\\&K &                                Iterative filtering &      epoch &              10-fold CV &      0.88 &   0.83 &      0.88 &      0.77 \\\\\n",
      "      hsu\\_2013 &  Sleep-EDF &  Fpz-Cz &         8 &                  1 &          R\\&K &       Energy features, recurrent neural classifier &      epoch &              10-fold CV &      0.90 &   0.82 &      0.90 &      0.77 \\\\\n",
      "          ours &     SHHS-1 &   C4-A1 &      5728 &                  1 &          R\\&K &                                                CNN &     record &             0.5/0.2/0.3 &      0.87 &   0.81 &      0.87 &      0.78 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(df.to_latex(index=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## More details stats for our results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas_ml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted       0      1       2       3       4  __all__\n",
      "Actual                                                   \n",
      "0          408881   7010   19800    3740    7904   447335\n",
      "1           11161  20614   18152      55    9047    59029\n",
      "2           16547   6425  582886   28899   18438   653195\n",
      "3            1816      7   30050  183631     336   215840\n",
      "4            8793   3477   20914     186  201983   235353\n",
      "__all__    447198  37533  671802  216511  237708  1610752\n"
     ]
    }
   ],
   "source": [
    "y_true_ours, y_pred_ours = get_y_true_pred(cm_ours)\n",
    "cm = pandas_ml.ConfusionMatrix(y_true_ours, y_pred_ours)\n",
    "print(cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PPV(Precision) for Wake, N1, N2, N3, REM, Total:\n",
      "0.914317595338\n",
      "0.549223350119\n",
      "0.867645526509\n",
      "0.848137046155\n",
      "0.84971056927\n",
      "0.863703362147\n",
      "TPR(Sensitivity=Recall) for Wake, N1, N2, N3, REM, Total:\n",
      "0.914037578101\n",
      "0.349218180894\n",
      "0.892361392846\n",
      "0.850773721275\n",
      "0.858212982201\n",
      "0.86791448963\n",
      "TNR(Specificity) for Wake, N1, N2, N3, REM, Total:\n",
      "0.967065119385\n",
      "0.989096636449\n",
      "0.907142864602\n",
      "0.976428620587\n",
      "0.974025719082\n",
      "0.945844470476\n"
     ]
    }
   ],
   "source": [
    "print('PPV(Precision) for Wake, N1, N2, N3, REM, Total:') \n",
    "print(cm.PPV)\n",
    "print('TPR(Sensitivity=Recall) for Wake, N1, N2, N3, REM, Total:') \n",
    "print(cm.TPR)\n",
    "print('TNR(Specificity) for Wake, N1, N2, N3, REM, Total:')\n",
    "print(cm.TNR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
